<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
    <meta charset="utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
      <link rel="shortcut icon" href="../img/favicon.ico" />
    <title>Openimages - webdataset</title>
    <link rel="stylesheet" href="../css/theme.css" />
    <link rel="stylesheet" href="../css/theme_extra.css" />
        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/github.min.css" />
    
      <script>
        // Current page data
        var mkdocs_page_name = "Openimages";
        var mkdocs_page_input_path = "openimages.md";
        var mkdocs_page_url = null;
      </script>
    
    <script src="../js/jquery-3.6.0.min.js" defer></script>
    <!--[if lt IE 9]>
      <script src="../js/html5shiv.min.js"></script>
    <![endif]-->
      <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
      <script>hljs.initHighlightingOnLoad();</script> 
</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">
    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
    <div class="wy-side-scroll">
      <div class="wy-side-nav-search">
          <a href=".." class="icon icon-home"> webdataset
        </a><div role="search">
  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
      <input type="text" name="q" placeholder="Search docs" title="Type search term here" />
  </form>
</div>
      </div>

      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
              <ul>
                <li class="toctree-l1"><a class="reference internal" href="..">Home</a>
                </li>
              </ul>
              <ul>
                <li class="toctree-l1"><a class="" href="../api/index.html">API</a>
                </li>
              </ul>
              <p class="caption"><span class="caption-text">Examples</span></p>
              <ul>
                  <li class="toctree-l1"><a class="reference internal" href="../cifar10/">CIFAR10</a>
                  </li>
              </ul>
      </div>
    </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="..">webdataset</a>
        
      </nav>
      <div class="wy-nav-content">
        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href=".." class="icon icon-home" alt="Docs"></a> &raquo;</li>
      <li>Openimages</li>
    <li class="wy-breadcrumbs-aside">
          <a href="http://github.com/webdataset/webdataset/edit/master/docs/openimages.md" class="icon icon-github"> Edit on GitHub</a>
    </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div class="section" itemprop="articleBody">
              
                <pre><code class="language-python">import sys
sys.path.insert(0, &quot;..&quot;)
</code></pre>
<pre><code class="language-python">import webdataset as wds
from torchvision import transforms
from pprint import pprint

bucket = &quot;pipe:curl -s -L http://storage.googleapis.com/nvdata-openimages/&quot;
# shards_train = bucket + &quot;openimages-train-{000000..000554}.tar&quot;
shards_train = bucket + &quot;openimages-train-{000000..000009}.tar&quot;

train_transform = transforms.Compose(
    [
        transforms.RandomHorizontalFlip(),
        transforms.RandomResizedCrop((32, 32), scale=(0.8, 1.0), ratio=(0.9, 1.1)),
        transforms.ToTensor(),
        # transforms.Normalize(DATA_MEANS, DATA_STD)
    ]
)

train_ds = wds.DataPipeline(
    wds.SimpleShardList(shards_train),
    wds.detshuffle(100),
    wds.cached_tarfile_to_samples(verbose=True),
    wds.decode(&quot;pil&quot;),
    wds.to_tuple(&quot;png;jpg json&quot;),
    wds.map_tuple(train_transform, None),
)

image, info = next(iter(train_ds))
print(image.shape)
print(info)
</code></pre>
<pre><code># downloading http://storage.googleapis.com/nvdata-openimages/openimages-train-000000.tar to ./_cache/nvdata-openimages/openimages-train-000000.tar


torch.Size([3, 32, 32])
[{'ImageID': 'e39871fd9fd74f55', 'Source': 'xclick', 'LabelName': '/m/01g317', 'Confidence': '1', 'XMin': '0.389323', 'XMax': '0.661458', 'YMin': '0.094727', 'YMax': '0.571289', 'IsOccluded': '1', 'IsTruncated': '0', 'IsGroupOf': '0', 'IsDepiction': '1', 'IsInside': '0'}, {'ImageID': 'e39871fd9fd74f55', 'Source': 'xclick', 'LabelName': '/m/05y5lj', 'Confidence': '1', 'XMin': '0.402344', 'XMax': '0.427083', 'YMin': '0.468750', 'YMax': '0.485352', 'IsOccluded': '1', 'IsTruncated': '0', 'IsGroupOf': '0', 'IsDepiction': '1', 'IsInside': '0'}, {'ImageID': 'e39871fd9fd74f55', 'Source': 'xclick', 'LabelName': '/m/05y5lj', 'Confidence': '1', 'XMin': '0.516927', 'XMax': '0.597656', 'YMin': '0.093750', 'YMax': '0.153320', 'IsOccluded': '0', 'IsTruncated': '0', 'IsGroupOf': '0', 'IsDepiction': '1', 'IsInside': '0'}]


# opening ./_cache/nvdata-openimages/openimages-train-000000.tar
</code></pre>
<pre><code class="language-python">
</code></pre>
              
            </div>
          </div><footer>

  <hr/>

  <div role="contentinfo">
    <!-- Copyright etc -->
  </div>

  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
          
        </div>
      </div>

    </section>

  </div>

  <div class="rst-versions" role="note" aria-label="Versions">
  <span class="rst-current-version" data-toggle="rst-current-version">
    
        <span>
          <a href="http://github.com/webdataset/webdataset" class="fa fa-github" style="color: #fcfcfc"> GitHub</a>
        </span>
    
    
    
  </span>
</div>
    <script>var base_url = '..';</script>
    <script src="../js/theme_extra.js" defer></script>
    <script src="../js/theme.js" defer></script>
      <script src="../search/main.js" defer></script>
    <script defer>
        window.onload = function () {
            SphinxRtdTheme.Navigation.enable(true);
        };
    </script>

</body>
</html>
